{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/pinecone-io/examples/blob/master/learn/search/semantic-search/light-demo/light-demo.ipynb) [![Open nbviewer](https://raw.githubusercontent.com/pinecone-io/examples/master/assets/nbviewer-shield.svg)](https://nbviewer.org/github/pinecone-io/examples/blob/master/learn/search/semantic-search/light-demo/light-demo.ipynb)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8HixLmI9-x7r"
      },
      "source": [
        "We start by installing prerequisite libraries:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "executionInfo": {
          "elapsed": 17314,
          "status": "ok",
          "timestamp": 1636648891892,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "i_EvV9FZ-x7t",
        "outputId": "d08c9663-1102-427e-911f-f242fabf01e0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting sentence-transformers\n",
            "  Downloading sentence-transformers-2.1.0.tar.gz (78 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 78 kB 3.3 MB/s \n",
            "\u001b[?25hCollecting pinecone-client\n",
            "  Downloading pinecone_client-2.0.3-py3-none-any.whl (156 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 156 kB 11.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.9.0+cu111)\n",
            "Collecting datasets\n",
            "  Downloading datasets-1.15.1-py3-none-any.whl (290 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 290 kB 35.8 MB/s \n",
            "\u001b[?25hCollecting transformers<5.0.0,>=4.6.0\n",
            "  Downloading transformers-4.12.3-py3-none-any.whl (3.1 MB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 3.1 MB 35.3 MB/s \n",
            "\u001b[?25hCollecting tokenizers>=0.10.3\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 3.3 MB 45.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (4.62.3)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (0.10.0+cu111)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (1.19.5)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (0.22.2.post1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (1.4.1)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (from sentence-transformers) (3.2.5)\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.96-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1.2 MB 51.3 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub\n",
            "  Downloading huggingface_hub-0.1.2-py3-none-any.whl (59 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 59 kB 4.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.10.0.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (3.3.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (2019.12.20)\n",
            "Collecting sacremoses\n",
            "  Downloading sacremoses-0.0.46-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 895 kB 38.6 MB/s \n",
            "\u001b[?25hCollecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 596 kB 55.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (21.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (4.8.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers<5.0.0,>=4.6.0->sentence-transformers) (2.23.0)\n",
            "Requirement already satisfied: pyparsing<3,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers<5.0.0,>=4.6.0->sentence-transformers) (2.4.7)\n",
            "Requirement already satisfied: urllib3>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from pinecone-client) (1.24.3)\n",
            "Collecting sentry-sdk>=1.0.0\n",
            "  Downloading sentry_sdk-1.4.3-py2.py3-none-any.whl (139 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 139 kB 75.5 MB/s \n",
            "\u001b[?25hCollecting loguru>=0.5.0\n",
            "  Downloading loguru-0.5.3-py3-none-any.whl (57 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 57 kB 3.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.7/dist-packages (from pinecone-client) (2.8.2)\n",
            "Collecting dnspython>=2.0.0\n",
            "  Downloading dnspython-2.1.0-py3-none-any.whl (241 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 241 kB 59.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.5.3->pinecone-client) (1.15.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=4.6.0->sentence-transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=4.6.0->sentence-transformers) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers<5.0.0,>=4.6.0->sentence-transformers) (2.10)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 1.1 MB 53.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 243 kB 59.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyarrow!=4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Collecting fsspec[http]>=2021.05.0\n",
            "  Downloading fsspec-2021.11.0-py3-none-any.whl (132 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 132 kB 60.5 MB/s \n",
            "\u001b[?25hCollecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (192 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 192 kB 59.5 MB/s \n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.1-py3-none-any.whl (5.7 kB)\n",
            "Collecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Collecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.2.0)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 271 kB 39.4 MB/s \n",
            "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-5.2.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (160 kB)\n",
            "\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 160 kB 59.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers<5.0.0,>=4.6.0->sentence-transformers) (3.6.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<5.0.0,>=4.6.0->sentence-transformers) (1.1.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<5.0.0,>=4.6.0->sentence-transformers) (7.1.2)\n",
            "Requirement already satisfied: pillow>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision->sentence-transformers) (7.1.2)\n",
            "Building wheels for collected packages: sentence-transformers\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-2.1.0-py3-none-any.whl size=121000 sha256=04fb9081810b6efbcc302ced0252ce4b8465990bbc373a8ca4d2e8456feba3fe\n",
            "  Stored in directory: /root/.cache/pip/wheels/90/f0/bb/ed1add84da70092ea526466eadc2bfb197c4bcb8d4fa5f7bad\n",
            "Successfully built sentence-transformers\n",
            "Installing collected packages: multidict, frozenlist, yarl, pyyaml, asynctest, async-timeout, aiosignal, tokenizers, sacremoses, huggingface-hub, fsspec, aiohttp, xxhash, transformers, sentry-sdk, sentencepiece, loguru, dnspython, sentence-transformers, pinecone-client, datasets\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed aiohttp-3.8.0 aiosignal-1.2.0 async-timeout-4.0.1 asynctest-0.13.0 datasets-1.15.1 dnspython-2.1.0 frozenlist-1.2.0 fsspec-2021.11.0 huggingface-hub-0.1.2 loguru-0.5.3 multidict-5.2.0 pinecone-client-2.0.3 pyyaml-6.0 sacremoses-0.0.46 sentence-transformers-2.1.0 sentencepiece-0.1.96 sentry-sdk-1.4.3 tokenizers-0.10.3 transformers-4.12.3 xxhash-2.0.2 yarl-1.7.2\n"
          ]
        }
      ],
      "source": [
        "!pip install sentence-transformers pinecone-client torch datasets sacremoses"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "71ehTJ-t-x7u"
      },
      "source": [
        "There are many sentence transformer models covering paraphrasing, Q&A, text-image, and in our case semantic similarity. Pretrained models can be found from [here](https://sbert.net/docs/pretrained_models.html). We download and initialize a model instance like so:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 497,
          "referenced_widgets": [
            "72dd3990f51f4d55b3643caa922efd98",
            "8f68ff7afff541f0b49c0ff23fcd0082",
            "81f5f778605c428fa1611b13754af5fa",
            "7f626c482f354fb88e9336380ca61b7c",
            "bfff8f67bef5422a922a223b83ecfe28",
            "125657af017f43b7a3df7c639bc884a0",
            "a6781580b7d74ee6b2917e603ecf58b4",
            "102b7ce9f5404d3b8ce1e39ec8639e9f",
            "86629dc4441040e08d43f1f88038e1bb",
            "9f310887e72840aaaf34d45799bfc353",
            "662ca18e164342fcad2b1addc99eed2e",
            "12085804b07d4a9cb4a151add387f1e3",
            "935a1e33bd3a4eab9189047d32bdbcf4",
            "91e2b719b308455f81f7765eeba0db0d",
            "5156296d37ac477ab459d7ec4ca23d16",
            "2a8fefbb4c664cf9a0f8847a1435b6c8",
            "80f6d57bbb6c41618e05d9d79256dcc6",
            "04544eaa547d49cea8b2e24ef5ddf9db",
            "792177a04cdf4042901e56fdb5025d2f",
            "745fc43a86f74fc68d26bec4ea4b0ded",
            "36ddede2fd5a40158b09c62e5117e8db",
            "87b79df72aa54df39ad9ba530b18aa3f",
            "e485322ef22e4a7783ea7fff0377f669",
            "df8f3d377bc641d69d00cf835e0662dd",
            "43e64b01b6fa483f9bb3936bda8a13ca",
            "c63e637fa8c64800ad4d91c6e47889df",
            "686e5cc3038641ee8d37a18ac0d8a25d",
            "b9638c933cdd4d5ebd2ad046541133dc",
            "143847dbe8f74b52bb6d23e684b914ea",
            "92cd60ba7a1b454abce2a5b3fc742cf0",
            "b8736f916f884dfc90202ee795715e73",
            "38e230fb663242a49fa60b21ecd5e3dd",
            "78bd2d69ccde4155af354326c3ce513b",
            "7fd1181e3cb742b2a055e156da7e8a79",
            "095f540868854278b5db1120620976aa",
            "0f829f4271204d50b42dd8514f64985c",
            "76a6ff619917441694192fb80e3a93b9",
            "5ffac6688b124869945e149aa942040a",
            "a26d16a0739d43c486598da74ea51cfc",
            "5d2656cbe4fa41a296d63153c1eca141",
            "d4213142e614427eb293e594392754e4",
            "b147a6ca30b94f2fa8dd616cad8bbe30",
            "651bac3629eb4156917e2cd3ac2f42a6",
            "8042966d3f324fddb5a0a67f7e777c30",
            "71b9da13672946798e70302950d0ba03",
            "6794f1c8d2684fdfa48f1f52740822b1",
            "16a38bb7e37f43b5a2d4fad04b323db1",
            "73eeb19a87b643cea9facf26a8061348",
            "d45ef5c1107a4634b1efeabb8d6613d1",
            "ab95e3c8d58349d19a1781fd00fe104a",
            "2b2d52cd5f1e4faea473e8c023b677e3",
            "8c9a5a827e354349a8f98bff1662d108",
            "04adfd804cde42dd9f1792a0b5d74f0a",
            "58314fa32d464fc99987d3cf7677eb37",
            "7edd6b6dde91497ab03c2997f8102c73",
            "82ee1cf8a21e48168fbc41a16dfe88bb",
            "c8501b3eaee644ab91f8679638a7c2e5",
            "001229be55684d238caaff5ad9c81b95",
            "e2b4429630104bddb1f186798e0bae89",
            "2cedd11f074b4d45a2f9220ff5c0ab7d",
            "ba0e0a44f111448ab529ef987a55c25f",
            "349cee03cb414837aebeeaa7c60b93a7",
            "f34a953647a84f21a7d343a831b99da4",
            "fa7db92cd52d42138ec5ae8bea6fab60",
            "e4e61e0e13d54084be5db7c6116fe485",
            "442fe510e5c04602b1833fbba8259bcf",
            "054df11c59f640f5838cfba470a47074",
            "9f234854fbe3488e918982a220fdbc5f",
            "ae3b7f68af59437eaed662654a10b4c1",
            "252ee375e8a2425b9222a2d86ccb7c48",
            "4997a10ae66a458a99295d28e8b57a9a",
            "31137d6305ad4e8cbf8b3ec234abca00",
            "ef85cd3d94f8431faf1034b7ca2692a6",
            "90b0e6a9e19747d7a4ae8836a95aaa6b",
            "dfa5efbd8625481d80bc05d4fb5e4d2a",
            "1e8e41073d27455d98b1003d9a6f77f8",
            "34a9726dc909425087487023829720be",
            "d5eb8ce8d9bf47dd99e5c473e7b84d52",
            "b6177e1071374169b45a997a86607966",
            "b6fe9587cdbb4467bcc792ef0f551006",
            "d89737375f774f3cb95f452b9bc7ba67",
            "7885f5d7ce154d34981ac186f79d1d4c",
            "e44544925fc94b0980529da3e809893c",
            "ff439b8c63de458f9da23f7e9035f31f",
            "6f2e34cfe1cd416689574a5a4f5b3ba6",
            "6ae6b03224de47018b6fb45fd20b4a07",
            "5bd19e575d0248bb898063cacb45e473",
            "2d7e4154f3d44c09aba370f000d7783b",
            "3b7b619db38a4fb8868006e4760e4562",
            "a866c35eed8d49d1883e0f02e884af0b",
            "0c0e62d1cde14d87b273350c9e36f99f",
            "724b40cdbd054b3f8eef31fc49b59906",
            "f4145601ff584456bbfd6db057a067af",
            "49e2ffcb8886496794a888be61ce9f73",
            "25c16a57d2544bf6847d58916457c92e",
            "6994a319d7e84fc7b09b3d62ba22a9cd",
            "7edad4d063cd4218b6409b0cbfb8ddda",
            "b1b634886d1a4599927f659a16ed5d59",
            "bd6fbe8bf149498d8e202518fa999074",
            "a246a8be007049f182ce7b81c2d2b1c9",
            "161407976aeb4feca46093afc9cb61ce",
            "4a44b5ecf9e44286ae44ee0f22dff412",
            "dba1ed2aa8ec42138b3976a82b938ca9",
            "d52c3688d8624f51b4519df6982fdffa",
            "cb3149d82651459f95320213944c8ab0",
            "c7251791f31349c99af5132440eed4c7",
            "98f818d52aa5417e95959de6ef1d4a52",
            "d4b5f0403404481ba78edc3e3d4549da",
            "3dd48f7690c6467b9e9df07c5fad4912",
            "95c0308d0c124af587b05845115c1516",
            "acb98d8d1c5142c5bdcd3cf55371df43",
            "476840df222a463ab56a9602a8edf48e",
            "3ee2940c85d64762b995f05283457c40",
            "04f56c27522f4f7d93ba4fb6d699b7fb",
            "cb2c4d7d08f743e7a7c0389bfc7da29c",
            "ab8a4a03bb9e47aca3eaa8d69773fbc8",
            "e5be059bcceb407a9b26733ed2c74dc5",
            "57c770300f8e4cd0a74803f34d05a09f",
            "6133c9549319404bb55aa2ffb16601d7",
            "1b8a22d7694448969fd0b5dc1431c5d3",
            "2a966dd37cae4f2baf8de05d4e44e381",
            "7cb3c7ce9c844351903fb8fb3e6d100a",
            "c6a2ac7a14734e3998ccd4ab161bb0ce",
            "2c0c725705cf4034a6ae4aa2df030577",
            "6a76a8c01bdf412bb379bfae26c48b74",
            "230cea9c5342407da473c2e8c96aa90d",
            "2c343a37b9df4029bee5d192c38eda80",
            "1b64708818374d46ace93f97fe07e8b4",
            "6f1475af0a3f40398a357c6dba554848",
            "c4d3df42b3a0471cb8766a6f0e2ca3ca",
            "9c04d5391e084c9993f8fc6cf5e2238e",
            "0fb7e5e343864014b0cb76a3ef7044b4",
            "0259cf39a4f8486689c8368d8bfb7259",
            "a60a872e0316400fac8274f7528fa9f8",
            "380eb39a4bd64eb08595b2e060ebd542",
            "4ef970ce95ad4bd5a841a4635ade09e8",
            "df99c748565f41a3b8303294c4e3838d",
            "da301fcdf6424a14949b1903abde4807",
            "959e4779f161400f995cb7d67c7f8baf",
            "493d38f871a74b8ab2cb8673c70150f0",
            "0000a0e5651c4e0a91da4252df77cbd4",
            "1defa63d966b4ae683bc857144896696",
            "1b52ac9ad6594402a37c087ec5aad54e",
            "81e2172d4c8348b9879ca37f22cf845a",
            "c975ac968597467395616a774b60be75",
            "b32feb347d8e422a8a1118d5748b466c",
            "e36f6d82050f48cd93837d5da64b0766",
            "9c12a2c939bf499487caccc17f777361",
            "190e040af4224e4aadd2ae8236f3dbc3",
            "b850d0e982a24fda99e19325c9b6fa25",
            "f7c761cdf4544340bf7f26108a96ad42",
            "154156acbea846ba889603090a4f20f7",
            "3fa8274e084549f3a0ab2f12ddeb9a4c",
            "e0b2d6d26115489699dcf25d9c5f2f25",
            "6de7f57f6eeb4ea08401a0da52097d3f",
            "3ce870f21812469d8e35ad8f01281df5",
            "2824fd74780a4d6d93374baea64a62be",
            "d93d49eab2f0499fa14f2b020fcc704e",
            "9b56888f3b4e49ab9b201eee96f2aa34",
            "79a6fa38da3c4145bf6cbfc70087303f",
            "24b71b64061e43ef9a3abb580bceffef",
            "2d736fd9f0f64097b953dd9bd407019a",
            "516d0ebdaf794cf898af87ee77b4328a",
            "e420e0bee6724810aa6d49b0eac4ae19",
            "06ef02d405ea495d962d0db87f8da0b3"
          ]
        },
        "executionInfo": {
          "elapsed": 44961,
          "status": "ok",
          "timestamp": 1636648947583,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "DoisQKvA-x7u",
        "outputId": "b8e7cca0-250f-45d5-96af-e4445beb48f0"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "72dd3990f51f4d55b3643caa922efd98",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/737 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "12085804b07d4a9cb4a151add387f1e3",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/9.86k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e485322ef22e4a7783ea7fff0377f669",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/653 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7fd1181e3cb742b2a055e156da7e8a79",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/116 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "71b9da13672946798e70302950d0ba03",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/15.7k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "82ee1cf8a21e48168fbc41a16dfe88bb",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "054df11c59f640f5838cfba470a47074",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/349 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d5eb8ce8d9bf47dd99e5c473e7b84d52",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/329M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3b7b619db38a4fb8868006e4760e4562",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/53.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a246a8be007049f182ce7b81c2d2b1c9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/239 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "acb98d8d1c5142c5bdcd3cf55371df43",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7cb3c7ce9c844351903fb8fb3e6d100a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/333 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0259cf39a4f8486689c8368d8bfb7259",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/13.1k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "81e2172d4c8348b9879ca37f22cf845a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/798k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6de7f57f6eeb4ea08401a0da52097d3f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/190 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import torch\n",
        "from sentence_transformers import SentenceTransformer\n",
        "\n",
        "# set device to GPU if available\n",
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "model = SentenceTransformer('all-distilroberta-v1', device=device)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jlU0QWq0-x7v"
      },
      "source": [
        "We'll encode some example sentences and work through the process of *upserting* those to Pinecone.\n",
        "\n",
        "We will use the Quora question duplicates dataset, which contains pairs of questions are not syntactically the same, but share the same meaning. We use HuggingFace's `datasets` library to access the dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 316,
          "referenced_widgets": [
            "93061962c599439eac42b2bb00478461",
            "57f469d007d645a0ba33f5b8b60a5a57",
            "bb7fc5ccbb084f71b74436f79f42353c",
            "cc90edacfc334f968d8c9adf566cd97d",
            "b4dd12e961b54643a6cb4c1b248147cb",
            "4372f71855da494e999c558985a8f845",
            "1a109b1bef8b449ab0b81b7ef4713de8",
            "b361a04d48d64e06b340d12325b73c53",
            "cb348d4828174fb99009770329b3289b",
            "2bdd4a7a492b498eabe539720730d7be",
            "279845af60404aca9d53d1f2bc531915",
            "10ceb82692884dbcbc5264e706d9c081",
            "3aa1e32e5eda447d928a56db8e57ab06",
            "7cfb4df7c3ef48ecb98c107e4e6577f6",
            "f7e507042b7a4338bf28b7780a6daad2",
            "fb1cdc35f7ca4fa48ec4f3f1ae8b99b2",
            "bda552a417734684b3ffebe11e40e07c",
            "ce4fbf818a84444e81c038750af8b5ba",
            "a19c1cafd0d74629adc7e4f81a977f32",
            "1a4297b8277741588f10189e53ce85fa",
            "dd4222bcea85407bb30bd0c48baca736",
            "02bfbfd9f6284b0495497e52d5af7ee7",
            "af0ecc0504d44cbbb02b61e78c09da24",
            "d20b47c9a6854a1facde81a38fd6883c",
            "6b7f4f471bed42e58c4682b97d3faa20",
            "f3798a5280184963b1bbb24be03efd82",
            "dd1ba6e78f8c451ca671e79d22be23cb",
            "f4e0c9f61b804be2a442c2661378aadd",
            "e4383a0b52e243db93862b00fc802731",
            "f040a281b2844eafb6d776cb7e608c50",
            "93b8653bb71d4329b1e24175812b74ef",
            "877e2a48579046158fcc8b698a46f1f3",
            "4a2401287aa14d058ea252330cd55777",
            "1f98028728904b1cb267ffb75d2a1490",
            "1e338e38830547e09cadb70a88242eb4",
            "cea3dd5954b449a69defe36998a4c6f2",
            "fd5a7e9aa3684cb2b7b000326757debd",
            "533a06d9235c4429a17380f282f216bf",
            "d0a9f689ab9d4995a7a395af3fe140c5",
            "a9150cc550b14c999390959b26d1da7c",
            "1f34bd3999614b58852d67201e175808",
            "3cc1ef38dcc0410c9e42e08cfbeae4d2",
            "40da33aa31d64e499ed60fe885c3d2ce",
            "fbdbfdb9286d4758a6f581f89c778cc0",
            "6b1f2186ccd241c3b7918a8027f588fb",
            "59a10ca70ed84c9088a0f4e40a35087c",
            "d43c1323d4224f619d0e37eae56394f2",
            "696ee4c3dc2c4f9db5ed06573b3683c9",
            "c517c684cfb347619281baeb2b46ba84",
            "b2579d0e0ea443bdb60a785926b324b0",
            "91e608402ce043a2a22e140c1672527d",
            "e5d5c7f0f52647f69cdd1972f78e0654",
            "73b86963686f46da8fb832d54acb049c",
            "56420d449b7844c8acc3b5bd5acf01f6",
            "1e09078104ba40ebb8ba837f2b894aef",
            "164e0670992d49be85c6967173b6a35b",
            "ec80b3b99fd048f7a0a33cb4c616e609",
            "1bb3d3d74def4a75a4e51521ad9ffc39",
            "3c1566b736864741955b97f5ad80af99",
            "c0ae5bf47f154e1cab139e28781a7bc2",
            "70bcf4c30a1b4b19bd4b256d9afa6156",
            "303cc2478552409488c1f7c04c64b024",
            "193e942dbf4a40e08534352bdc08d39b",
            "695c724a04dc49f68eb6d021388844d2",
            "563c01b63031411fa2d6701ae77ae4af",
            "9fd0b89e7340462ab9bc0bebf4273759"
          ]
        },
        "executionInfo": {
          "elapsed": 35456,
          "status": "ok",
          "timestamp": 1636648984168,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "ymzcE1iK-x7v",
        "outputId": "69dd8569-3fbf-4c9e-daf8-c1c2a31fe5cd"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93061962c599439eac42b2bb00478461",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.06k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "10ceb82692884dbcbc5264e706d9c081",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/559 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Using custom data configuration default\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading and preparing dataset quora/default (download: 55.48 MiB, generated: 55.46 MiB, post-processed: Unknown size, total: 110.94 MiB) to /root/.cache/huggingface/datasets/quora/default/0.0.0/36ba4cd42107f051a158016f1bea6ae3f4685c5df843529108a54e42d86c1e04...\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "af0ecc0504d44cbbb02b61e78c09da24",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1f98028728904b1cb267ffb75d2a1490",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/58.2M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6b1f2186ccd241c3b7918a8027f588fb",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "164e0670992d49be85c6967173b6a35b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dataset quora downloaded and prepared to /root/.cache/huggingface/datasets/quora/default/0.0.0/36ba4cd42107f051a158016f1bea6ae3f4685c5df843529108a54e42d86c1e04. Subsequent calls will reuse this data.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['questions', 'is_duplicate'],\n",
              "    num_rows: 300\n",
              "})"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import datasets\n",
        "\n",
        "quora = datasets.load_dataset('quora', split='train[:300]')  # we only include the first 300 samples in this notebook\n",
        "# the full dataset contains 404K pairs\n",
        "quora"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "executionInfo": {
          "elapsed": 229,
          "status": "ok",
          "timestamp": 1636648997201,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "ZF5udvvK-x7v",
        "outputId": "692a5481-41b2-4e41-e2ef-791492f60047"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'is_duplicate': False,\n",
              " 'questions': {'id': [1, 2],\n",
              "  'text': ['What is the step by step guide to invest in share market in india?',\n",
              "   'What is the step by step guide to invest in share market?']}}"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "quora[0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HygiB801-x7v"
      },
      "source": [
        "The full dataset contains >404K pairs, encoding all of these at once in-memory is not efficient so we will work through the data in batches and upsert them to Pinecone as we go. We will be upserting each sample as a tuple `(id, vectors, metadata)`, which each contain:\n",
        "\n",
        "* `id` - a str ID\n",
        "\n",
        "* `vectors` - the sentence vector (in list format)\n",
        "\n",
        "* `metadata` - a dictionary in the format:\n",
        "\n",
        "```json\n",
        "{\n",
        "    'tokens': <list of tokens for keyword search>,\n",
        "    'is_duplicate': <True/False whether this is a duplicate question>,\n",
        "    'char_length': <length of sentence (in text characters)>\n",
        "}\n",
        "```\n",
        "\n",
        "To create `'vectors'` and `'tokens'` we need to use our sentence transformer `encode` method and a tokenizer respectively. The tokenizer will come from HuggingFace transformers and *should* break text into words like so:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 130,
          "referenced_widgets": [
            "24d14a00dc2a48a8a09860d0ee9b7dfb",
            "760c8cfc83264569a56f96dc7a2f70ed",
            "1696472bf4904762a7f83ca59231678d",
            "f66f0a9e54a24db2ac889d51574a34bf",
            "74a80d1302f540b5b98ca68a683382a4",
            "65df9f06db164521bb245f7c2daabcfb",
            "6326059b97934cbf9937e2ad3a26f284",
            "8af7e5197aeb48c3b9fc2ec74ca7ad10",
            "1de337f321724d2caab3897a26130363",
            "c6beecd9c2a5448b99f1cc4d1a8aa415",
            "3a33425371f843088636bd553aa70317",
            "99e7514527b24a5eb676dff78086501c",
            "1424c1a2175849029512db672f09ccfd",
            "42cd41752b5644eea5ef8fb722079b2f",
            "ffe935f7eb8a49eaa77c35be2a12cf82",
            "684368d9b8974b4c9bfbe28be8754712",
            "22e90b6b5bf04b2d915ed861ee013f47",
            "fa21a7d6570543a59b2fe9b63907ce46",
            "659380e933fc4729bdca354d86606ae0",
            "051141060e174323a161608306b6b2cd",
            "7468072d67164681ad3de84c6590f8bf",
            "c2a6dc00fee34bba98e49ed09136ee1c",
            "819b160ab965400c8d4043b3b83f999d",
            "c0995e1fbcfa49be9dd13d53bbd521bd",
            "b60fe1893ea34526ad42c7b7a789360c",
            "69de4b5f02204b95bececb8b8483f1fd",
            "530420a07d5e4725b0122fb1b8e7ae9a",
            "ce0758eee2904e4baff8518ee4f93009",
            "0c42651a143740afa072431ae21db65f",
            "e715fbb89bbf497baf8be56b8c53dae5",
            "17ef8f5733d24d5f8b373078f347f1bf",
            "5d18714702fc410fa7d0a84ac2a45a60",
            "547dc1822ccb46f59d31e60e65c64d21"
          ]
        },
        "executionInfo": {
          "elapsed": 3946,
          "status": "ok",
          "timestamp": 1636649005765,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "-e_TmQ59-x7w",
        "outputId": "e1ca6b17-4524-40f8-9a73-8c6f93fa1ee6"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "24d14a00dc2a48a8a09860d0ee9b7dfb",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/856 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "99e7514527b24a5eb676dff78086501c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/8.72M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "819b160ab965400c8d4043b3b83f999d",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/8.72M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/plain": [
              "['purple', 'is', 'the', 'best', 'city', 'in', 'the', 'forest']"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "# transfo-xl tokenizer uses word-level encodings\n",
        "tokenizer = AutoTokenizer.from_pretrained('transfo-xl-wt103')\n",
        "\n",
        "tokenizer.tokenize('Purple is the BEST city in the forest'.lower())"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "DNj-FpTO-x7w"
      },
      "source": [
        "We will be processing and upserting all in one go. To upsert to Pinecone we would need to create an index to upsert to. We will do this via the Pinecone Python client. First we initialize our connection to Pinecone, this does require a [free API key](https://app.pinecone.io). You can find your environment in the [Pinecone console](https://app.pinecone.io) under **API Keys**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "executionInfo": {
          "elapsed": 686,
          "status": "ok",
          "timestamp": 1636649024105,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "OH796f5b-x7w"
      },
      "outputs": [],
      "source": [
        "from pinecone import Pinecone\n",
        "pinecone.init(\n",
        "    api_key='YOUR_API_KEY',\n",
        "    environment=\"YOUR_ENV\"  # find next to API key in console\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sFGon11828X7"
      },
      "source": [
        "Create a new index..."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "executionInfo": {
          "elapsed": 10939,
          "status": "ok",
          "timestamp": 1636649036403,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "wUBuSj5E21y4"
      },
      "outputs": [],
      "source": [
        "pinecone.create_index(name='search-webinar', dimension=768)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qFGYP0de22Qb"
      },
      "source": [
        "Then we connect to the index with:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "executionInfo": {
          "elapsed": 183,
          "status": "ok",
          "timestamp": 1636649039768,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "z1KwnKeq-x7x"
      },
      "outputs": [],
      "source": [
        "index = pinecone.Index('search-webinar')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5WhJ59RP-x7x"
      },
      "source": [
        "Now we can put all of this together, we will process our data in batches creating the *vectors* and *metadata* and upserting them to Pinecone as we go."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "7637f6d4fb6f4489995b8b0cbb33f912",
            "34a70b0fcdbd4dc38d7c85cbd70c603a",
            "526073f6ad1f406f88c37dba23171c26",
            "d413bd95a2c941febc502bcba40e0319",
            "d7602350671944828589597ea0a880b5",
            "0b06e90230314f24bc6b7e87cc84f512",
            "d0814dde4cfc42d48775dd0b42abdbb6",
            "008a6bca1c1f4a3d84365201a4e42ca5",
            "bd6d40d37fbe4774ab0c7de5b9f7138d",
            "12baeb44907e4e5dbcec13c3c78f243a",
            "2372ceac742d45db88ad0ac9be2813c3"
          ]
        },
        "executionInfo": {
          "elapsed": 38248,
          "status": "ok",
          "timestamp": 1636649083071,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "8CFnRKwu-x7x",
        "outputId": "9a723373-3710-4495-a974-e63ff7085d46"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7637f6d4fb6f4489995b8b0cbb33f912",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/300 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from tqdm.auto import tqdm  # progress bar\n",
        "\n",
        "data = []\n",
        "\n",
        "# loop through and create JSON files\n",
        "for i, row in enumerate(tqdm(quora)):\n",
        "    # each Quora row contains a pair of sentences, loop through both\n",
        "    for pair in [0, 1]:\n",
        "        text = row['questions']['text'][pair]\n",
        "        # append the (id, vectors, metadata) tuple to our 'data' list\n",
        "        data.append((\n",
        "            str(row['questions']['id'][pair]),\n",
        "            model.encode(text).tolist(),\n",
        "            {\n",
        "                'tokens': tokenizer.tokenize(text.lower()),\n",
        "                'is_duplicate': int(row['is_duplicate']),\n",
        "                'char_length': len(text)\n",
        "            }\n",
        "        ))\n",
        "    # once we reach end of dataset OR 100 samples, upsert to Pinecone\n",
        "    if len(data) == 100 or i == len(quora):\n",
        "        index.upsert(vectors=data)\n",
        "        # and now reset the data list\n",
        "        data = []"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qNz2apWe-x7y"
      },
      "source": [
        "## Querying with Pinecone\n",
        "\n",
        "We have our index and data ready-to-go - let's move onto querying. First we need to create a *'query vector'* `xq`. This is a sentence (or in this case question) encoded using the same model that we encoded the quora dataset with.\n",
        "\n",
        "*(if you are not running the full dataset - this will not return the same results! You can try the full dataset by removing `[:100]` in the `split` of `load_dataset` near the start of the notebook)*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "executionInfo": {
          "elapsed": 206,
          "status": "ok",
          "timestamp": 1636649184106,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "6kgrNQRl-x7y"
      },
      "outputs": [],
      "source": [
        "query = \"which Quora queries are good?\"\n",
        "xq = model.encode([query]).tolist()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-OX4d3o8-x7y"
      },
      "source": [
        "With this, we can return similar sentences using the `query` method."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "executionInfo": {
          "elapsed": 226,
          "status": "ok",
          "timestamp": 1636649187128,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "QuQui1iO-x7y",
        "outputId": "86999a5c-50c8-484a-ef7e-21a0381a5180"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'results': [{'matches': [{'id': '46',\n",
              "                           'metadata': {'char_length': 37.0,\n",
              "                                        'is_duplicate': 0.0,\n",
              "                                        'tokens': ['which',\n",
              "                                                   'question',\n",
              "                                                   'should',\n",
              "                                                   'i',\n",
              "                                                   'ask',\n",
              "                                                   'on',\n",
              "                                                   'quora',\n",
              "                                                   '?']},\n",
              "                           'score': 0.546800077,\n",
              "                           'values': []},\n",
              "                          {'id': '45',\n",
              "                           'metadata': {'char_length': 47.0,\n",
              "                                        'is_duplicate': 0.0,\n",
              "                                        'tokens': ['what',\n",
              "                                                   'are',\n",
              "                                                   'the',\n",
              "                                                   'questions',\n",
              "                                                   'should',\n",
              "                                                   'not',\n",
              "                                                   'ask',\n",
              "                                                   'on',\n",
              "                                                   'quora',\n",
              "                                                   '?']},\n",
              "                           'score': 0.466334224,\n",
              "                           'values': []},\n",
              "                          {'id': '319',\n",
              "                           'metadata': {'char_length': 40.0,\n",
              "                                        'is_duplicate': 1.0,\n",
              "                                        'tokens': ['why',\n",
              "                                                   'nobody',\n",
              "                                                   'answer',\n",
              "                                                   'my',\n",
              "                                                   'questions',\n",
              "                                                   'in',\n",
              "                                                   'quora',\n",
              "                                                   '?']},\n",
              "                           'score': 0.391543567,\n",
              "                           'values': []},\n",
              "                          {'id': '38',\n",
              "                           'metadata': {'char_length': 73.0,\n",
              "                                        'is_duplicate': 1.0,\n",
              "                                        'tokens': ['why',\n",
              "                                                   'do',\n",
              "                                                   'people',\n",
              "                                                   'ask',\n",
              "                                                   'quora',\n",
              "                                                   'questions',\n",
              "                                                   'which',\n",
              "                                                   'can',\n",
              "                                                   'be',\n",
              "                                                   'answered',\n",
              "                                                   'easily',\n",
              "                                                   'by',\n",
              "                                                   'google',\n",
              "                                                   '?']},\n",
              "                           'score': 0.355345547,\n",
              "                           'values': []},\n",
              "                          {'id': '320',\n",
              "                           'metadata': {'char_length': 46.0,\n",
              "                                        'is_duplicate': 1.0,\n",
              "                                        'tokens': ['why',\n",
              "                                                   'is',\n",
              "                                                   'no',\n",
              "                                                   'one',\n",
              "                                                   'answering',\n",
              "                                                   'my',\n",
              "                                                   'questions',\n",
              "                                                   'in',\n",
              "                                                   'quora',\n",
              "                                                   '?']},\n",
              "                           'score': 0.346606493,\n",
              "                           'values': []}],\n",
              "              'namespace': ''}]}"
            ]
          },
          "execution_count": 11,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "result = index.query(vector=xq, top_k=5, includeMetadata=True)\n",
        "result"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fQfcPkvy-x7z"
      },
      "source": [
        "We can use the ID values to map these back to the original sentences, we need to create a dictionary mapping IDs to text like so:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "executionInfo": {
          "elapsed": 218,
          "status": "ok",
          "timestamp": 1636649196512,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "kibqxTvi3hlB"
      },
      "outputs": [],
      "source": [
        "id2text = {}\n",
        "for row in quora:\n",
        "    for pair in [0, 1]:\n",
        "        id2text[str(row['questions']['id'][pair])] = row['questions']['text'][pair]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EK3XNwVKEtG5"
      },
      "source": [
        "Now we can map IDs to text."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "executionInfo": {
          "elapsed": 200,
          "status": "ok",
          "timestamp": 1636649198136,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "VwLB3YeI-x7z",
        "outputId": "5d2693e6-3251-4655-910f-e4767f37eaf2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.55\n",
            "Which question should I ask on Quora?\n",
            "0.47\n",
            "What are the questions should not ask on Quora?\n",
            "0.39\n",
            "Why nobody answer my questions in Quora?\n",
            "0.36\n",
            "Why do people ask Quora questions which can be answered easily by Google?\n",
            "0.35\n",
            "Why is no one answering my questions in Quora?\n"
          ]
        }
      ],
      "source": [
        "for item in result['matches']:\n",
        "    print(round(item['score'], 2))\n",
        "    print(id2text[item['id']])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KZlgzHrN-x7z"
      },
      "source": [
        "Let's try again but this time using metadata filtering to only return questions marked as *not* duplicates."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "executionInfo": {
          "elapsed": 490,
          "status": "ok",
          "timestamp": 1636649206845,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "a2EA7DwF-x7z",
        "outputId": "c2596f22-7717-4d45-85a1-02f62c30e10e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.55\n",
            "Which question should I ask on Quora?\n",
            "0.47\n",
            "What are the questions should not ask on Quora?\n",
            "0.23\n",
            "What is bestmytest.com?\n",
            "0.22\n",
            "What are the best quotes/lessons of the Assassin's Creed series?\n",
            "0.2\n",
            "What are the best YouTube channels to learn medicine?\n"
          ]
        }
      ],
      "source": [
        "result = index.query(vector=xq, top_k=5, filter={'is_duplicate': {'$eq': 0}})\n",
        "\n",
        "for item in result['matches']:\n",
        "    print(round(item['score'], 2))\n",
        "    print(id2text[item['id']])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KhjetPGs-x7z"
      },
      "source": [
        "Let's try adding a keyword search into this, let's see what appears when excluding the word 'Quora'."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "executionInfo": {
          "elapsed": 190,
          "status": "ok",
          "timestamp": 1636649217848,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "gR-BCsu5-x70",
        "outputId": "905d5c3c-e805-4de2-f887-d40d6aa97033"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.23\n",
            "What is bestmytest.com?\n",
            "0.22\n",
            "What are the best quotes/lessons of the Assassin's Creed series?\n",
            "0.2\n",
            "What are the best YouTube channels to learn medicine?\n",
            "0.2\n",
            "Which test series is the best for GATE computer science stream?\n",
            "0.2\n",
            "How can I ask a question without getting marked as \u2018need to improve\u2019?\n"
          ]
        }
      ],
      "source": [
        "result = index.query(vector=xq, top_k=5, filter={'tokens': {'$nin': ['quora']}})\n",
        "\n",
        "for item in result['matches']:\n",
        "    print(round(item['score'], 2))\n",
        "    print(id2text[item['id']])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "79-R4dm_-x70"
      },
      "source": [
        "Alternatively, we might change our query to be more generic - but then restrict the search to return questions containing one of several keywords."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "executionInfo": {
          "elapsed": 192,
          "status": "ok",
          "timestamp": 1636649231206,
          "user": {
            "displayName": "James Briggs",
            "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjOtZt5ezAHxOqj36--OeaLvoYzLBb90_HskJTRIQ=s64",
            "userId": "12893500451729195124"
          },
          "user_tz": 0
        },
        "id": "zNjYP2G4FLhv",
        "outputId": "5dcfa652-f03f-4ca1-f9d7-04416c4ef8eb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.06\n",
            "How Google helps in spam ranking adjustment of the search results?\n",
            "-0.01\n",
            "If I do not monetize YouTube videos & upload copyright content, then are there chances that Google may block my account?\n",
            "-0.02\n",
            "What is the distribution of traffic between Google organic search results? e.g. #1 vs. #2 in rankings, first page vs. second page\n"
          ]
        }
      ],
      "source": [
        "query = \"how to ask a good question?\"\n",
        "xq = model.encode([query]).tolist()\n",
        "\n",
        "result = index.query(vector=xq, top_k=5, filter={'tokens': {\n",
        "    '$nin': ['quora', 'quorans'],\n",
        "    '$in': ['google', 'reddit', 'stackoverflow']\n",
        "}})\n",
        "\n",
        "for item in result['matches']:\n",
        "    print(round(item['score'], 2))\n",
        "    print(id2text[item['id']])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qsi_ACXzHk51"
      },
      "source": [
        "And there is our demo on semantic search with sentence transformers and Pinecone."
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "light_demo.ipynb",
      "provenance": []
    },
    "interpreter": {
      "hash": "5188bc372fa413aa2565ae5d28228f50ad7b2c4ebb4a82c5900fd598adbb6408"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 0
}